{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBClassifier  \n",
    "from sklearn.model_selection import GridSearchCV,cross_val_score  \n",
    "from sklearn import  metrics  \n",
    "import matplotlib.pylab as plt  \n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import log_loss\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分别导入数据(对于二进制的数据,通过训练发现用于训练比较快,但在进行调参的时候优化时,由于不知道如何对二进制数据进行分割和可视化显示,只能采用 csv 数据)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/RentListingInquries_FE_train.csv')\n",
    "test=pd.read_csv('data/RentListingInquries_FE_test.csv')\n",
    "\n",
    "dtrain = xgb.DMatrix('data/RentListingInquries_FE_train.bin')\n",
    "dtest = xgb.DMatrix( 'data/RentListingInquries_FE_test.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "\n",
    "X_train = train.drop(['interest_level'], axis=1)\n",
    "\n",
    "dtrain_y= dtrain.get_label()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 采用 xgb.cv 确定一个最优 n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators:  163\n",
      "0.385361916989\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd8FHX++PHXezcdQk2kBSF0CKGGqoCAQJCm2LGAnu0EUe/Act5X77jys52nnoUDRSwgKCqioBSFExSFAAHpoRNBCCChpefz++OzCUsIsEDIbLLv58N57M5nZ2bfGdb3Z+Yzn/mMGGNQSikVGFxOB6CUUqr0aNJXSqkAoklfKaUCiCZ9pZQKIJr0lVIqgGjSV0qpAKJJXymlAogmfaWUCiCa9JVSKoAEOR1AUVFRUaZ+/fpOh6GUUmXKihUrDhhjos+1nN8l/fr165OUlOR0GEopVaaIyE5fltPmHaWUCiCa9JVSKoBo0ldKqQDid236SgWqnJwcUlNTyczMdDoU5cfCwsKIiYkhODj4gtbXpK+Un0hNTSUyMpL69esjIk6Ho/yQMYaDBw+SmppKbGzsBW1Dm3eU8hOZmZlUr15dE746IxGhevXqF3U2qElfKT+iCV+dy8X+RspN0jfG8M85G9iadszpUJRSym+Vm6S//cBxpi3bRf9XFvP6wi3k5OU7HZJSZcrhw4d54403Lmjdl19+mRMnTlzU948YMYIZM2Zc0LqTJ09m1KhRF/X9gaLcJP0G0RVZ8Mce9GlegxfmbmLQf5aQvPuw02EpVWY4nfRV6Sg3SR/gssgwXr+tHRPuaM/hEzkMfeN7/vrFOtIzcpwOTSm/98QTT7B161batGnD2LFjeeGFF+jQoQOtWrXimWeeAeD48eMMGDCA1q1b07JlS6ZPn86rr77Knj176NmzJz179jxtu5MnT+baa69l0KBBxMbG8tprr/HSSy/Rtm1bOnfuzKFDh05b55tvvqFt27bEx8dz9913k5WVBcCcOXNo1qwZV155JaNHj2bgwIGnrbtz50569+5Nq1at6N27N7t27QLg448/pmXLlrRu3Zru3bsDsG7dOjp27EibNm1o1aoVKSkpp23vL3/5C8OHD6dv377Ur1+fTz/9lMcee4z4+HgSExPJybH5ZcWKFfTo0YP27dvTr18/9u7dC8DEiRPp0KEDrVu35vrrry+sHEeMGMHo0aPp2rUrDRo0uOCznPNVLrts9o2rSZeG1Xnu641M/mEHnyfv4dGrG3Nrx8sJcperek6VU3/9Yh3r9xwp0W22qF2JZwbFnfHzZ599lrVr15KcnMy8efOYMWMGy5YtwxjD4MGD+e6770hLS6N27drMnj0bgPT0dCpXrsxLL73EwoULiYqKKnbba9euZdWqVWRmZtKoUSOee+45Vq1axaOPPsp7773HI488UrhsZmYmI0aM4JtvvqFJkybceeedvPnmmzzwwAPcf//9fPfdd8TGxnLrrbcW+12jRo3izjvvZPjw4UyaNInRo0czc+ZMxo0bx9y5c6lTpw6HD9tWgPHjx/Pwww9z2223kZ2dTV5eXrHb3Lp1KwsXLmT9+vV06dKFTz75hOeff57rrruO2bNnM2DAAB566CE+//xzoqOjmT59Ok899RSTJk1i6NCh3HvvvQD8+c9/5u233+ahhx4CYO/evSxZsoSNGzcyePBgbrjhhnP8K168cpsBI8OC+fu18Xwx6kqa1KjI/32+jv6vLGbRpv1Oh6aU35s3bx7z5s2jbdu2tGvXjo0bN5KSkkJ8fDwLFizg8ccfZ/HixVSuXNmn7fXs2ZPIyEiio6OpXLkygwYNAiA+Pp4dO3acsuymTZuIjY2lSZMmAAwfPpzvvvuOjRs30qBBg8L+6WdK+kuXLmXYsGEA3HHHHSxZsgSAK664ghEjRjBx4sTC5N6lSxf++c9/8txzz7Fz507Cw8OL3Wb//v0JDg4mPj6evLw8EhMTT4l/06ZNrF27lj59+tCmTRv+/ve/k5qaCtgKr1u3bsTHxzNlyhTWrVtXuN1rr70Wl8tFixYt2Ldvn0/78mKVyyN9by3rVObDezszb/0+/jlnAyPeWU6PJtE8NaA5TWpEOh2eUsU62xF5aTDG8OSTT3L//fef9tmKFSuYM2cOTz75JH379uXpp58+5fPPPvuMv/71rwC89dZbAISGhhZ+7nK5CuddLhe5ubmnffeZYroQBV0cx48fz08//cTs2bNp06YNycnJDBs2jE6dOjF79mz69evHW2+9xYYNG5g4cSJgm5O843e5XAQHBxdusyB+YwxxcXEsXbr0tO8fMWIEM2fOpHXr1kyePJlFixYVfua9Xy707ztf5fZI35uI0C+uJvMf7cGfBzRn5a7f6P/KYp789Gd2H9KLT0oBREZGcvToUQD69evHpEmTOHbMdoH+5Zdf2L9/P3v27CEiIoLbb7+dMWPGsHLlytPWve6660hOTiY5OZmEhITzjqNZs2bs2LGDLVu2APD+++/To0cPmjVrxrZt2wrPDKZPn17s+l27dmXatGkATJkyhSuvvBKwTTSdOnVi3LhxREVFsXv3brZt20aDBg0YPXo0gwcPZs2aNYwcObIw/tq1a/sUc9OmTUlLSytM+jk5OYVH9EePHqVWrVrk5OQwZcqU894fJa3cH+l7CwlycU+3BgxtF8PLCzbz4bJdfJS0m4GtavFAj4Y0r1XJ6RCVckz16tW54ooraNmyJf3792fYsGF06dIFgIoVK/LBBx+wZcsWxo4dW3jE++abbwJw33330b9/f2rVqsXChQsvKo6wsDDeeecdbrzxRnJzc+nQoQMPPPAAoaGhvPHGGyQmJhIVFUXHjh2LXf/VV1/l7rvv5oUXXiA6Opp33nkHgLFjx5KSkoIxht69e9O6dWueffZZPvjgA4KDg6lZs+ZpZy2+CgkJYcaMGYwePZr09HRyc3N55JFHiIuL429/+xudOnWiXr16xMfHF1aOTpHSOqXwVUJCgimth6j8mp7J20u2MfWnXRzPzuOqptHcfUUsVzaKwuXSOyNV6dqwYQPNmzd3Ogy/duzYMSpWrIgxhpEjR9K4cWMeffRRp8MqdcX9VkRkhTHmnKdWAdG8cyY1K4fx1IAW/PBEb8b0bcLPqencOWkZ3Z5fyCsLUthzOMPpEJVSXiZOnEibNm2Ii4sjPT292GsO6uwC+ki/qKzcPOat28dHSbtZnHIAEejRJJqh7WLo07wG4SFuR+JSgUGP9JWvLuZIP6Da9M8lNMjNoNa1GdS6NrsPneDjpN18vCKV0R+uomJoEIktazK0bR06N6iuzT9KqTLJp+YdEUkUkU0iskVEnjjDMjeJyHoRWSciU73Kh4tIimcaXlKBX2p1q0Xwh75N+f7xXky9txPXxNfk67W/Muytn+j67Lf8/cv1rEk9XGrdrJRSqiSc80hfRNzA60AfIBVYLiKzjDHrvZZpDDwJXGGM+U1ELvOUVwOeARIAA6zwrPtbyf8pl4bLJXRtGEXXhlGMG9KS+ev38XnyHt5duoO3lmwnNqoCg1rV4uoWNWhZu7KeASil/JovzTsdgS3GmG0AIjINGAKs91rmXuD1gmRujCm47bUfMN8Yc8iz7nwgEfiwZMIvXWHBJ5t/0k/k8NXavcxavYf/LNzCq99uoVqFELo1jqJ742i6NYnissgwp0NWSqlT+NK8UwfY7TWf6inz1gRoIiLfi8iPIpJ4HusiIveJSJKIJKWlpfkevYMqRwRzS8fLmXpvZ5Y/dTX/vrk1PZpE8/2WA/zx49V0/Mc39H9lMf/vqw38sPUA2bk61LPybxc6yuY111xTOJZNSbqY4ZJ37NhBy5YtSzii8sGXI/3i2iuKNmQHAY2Bq4AYYLGItPRxXYwxE4AJYHvv+BCTX4mqGMp1bWO4rm0M+fmG9XuP8L/NaXy3OY23F2/nv//bRkSImy4NqtO9STTdm0RTv3qEPiVJ+ZWCpP/ggw+eUp6Xl4fbfeaeawVDFaiywZcj/VSgrtd8DLCnmGU+N8bkGGO2A5uwlYAv65YrLpfQsk5lRvZsxPT7u5D8TF8m3pnA9e1i2JJ2jGdmraPni4vo/sJC/jzzZ+at+5VjWbnn3rBSl5j30ModOnSgZ8+eDBs2jPj4eMAODta+fXvi4uKYMGFC4Xr169fnwIED7Nixg+bNm3PvvfcSFxdH3759ycg4/V6XHTt20KxZM+655x5atmzJbbfdxoIFC7jiiito3Lgxy5YtO22dMw2XvHXrVjp37kyHDh14+umnqVix4mnrZmZmctdddxEfH0/btm0L7xgubljl4oaOLmrRokX06NGDm266iSZNmvDEE08wZcoUOnbsSHx8PFu3bgUgLS2N66+/ng4dOtChQwe+//57AJYtW0bXrl1p27YtXbt2ZdOmTYA9sxk6dCiJiYk0btyYxx577Lz+/XxmjDnrhD2K3wbEAiHAaiCuyDKJwLue91HYJp3qQDVgO1DVM20Hqp3t+9q3b2/Ksx0Hjpn3fthufjd5mWn+f1+Zeo9/aRo+OdvcOP4H89q3KSZpxyFzPCvH6TCVA9avX39yZs7jxky6pmSnOY+f9fu3b99u4uLijDHGLFy40ERERJht27YVfn7w4EFjjDEnTpwwcXFx5sCBA8YYY+rVq2fS0tLM9u3bjdvtNqtWrTLGGHPjjTea999/v9jvcbvdZs2aNSYvL8+0a9fO3HXXXSY/P9/MnDnTDBkyxBhjzDvvvGNGjhxpjDFm4MCBZvLkycYYY95+++3CZQYMGGCmTp1qjDHmzTffNBUqVDjtb3nxxRfNiBEjjDHGbNiwwdStW9dkZGSYUaNGmQ8++MAYY0xWVpY5ceKEmTFjhrnnnnsKYz18+PBp8S9cuNBUrlzZ7Nmzx2RmZpratWubp59+2hhjzMsvv2wefvhhY4wxt956q1m8eLExxpidO3eaZs2aGWOMSU9PNzk59v/x+fPnm6FDhxb+vbGxsebw4cMmIyPDXH755WbXrl3F/lud8lvxAJLMOfK5MebczTvGmFwRGQXMBdzAJGPMOhEZ5/mSWZ7P+orIeiAPGGuMOQggIn8Dlns2N854LuoGqnrVK3BHlwrc0aU+2bn5rNj5G9+l2KagF+baGl8EYqMq0KJWJVrUrkTzWpWIq1WJ6MhQbRJSpaZjx46FwxiDHdPms88+A2D37t2kpKRQvXr1U9aJjY2lTZs2ALRv3/60YZO9lys4g4iLi6N3796ISLFDLYMdLvnTTz8F7HDJBUfBS5cuZebMmQAMGzaMMWPGnLbukiVLCsevb9asGfXq1WPz5s106dKFf/zjH6SmpjJ06FAaN25MfHw8Y8aM4fHHH2fgwIF069at2Pg7dOhArVq1AGjYsCF9+/YF7FDLBWcSCxYsYP36k/1djhw5wtGjR0lPT2f48OGkpKQgIoUPYQHo3bt34XDVLVq0YOfOndSt691YcvF8ujnLGDMHmFOk7Gmv9wb4g2cquu4kYNLFhVk+hQS56NKwOl0aVufxxGakHc0iefdh1u85wvq96axOPcyXa/YWLl81IpiG0RVpdFlFGkZXpOFlFWgUHUmdquG4tato+dL/WacjoEKFCoXvFy1axIIFC1i6dCkRERFcddVVZGZmnraO91DBbrebjIwMdu/eXTh+/gMPPEBiYuJ5DbVcnPM5+DFnuJemuGGVe/XqddrQ0f369Ssc7mHcuHFUqlTJp/jz8/NZunTpaWP0P/TQQ/Ts2ZPPPvuMHTt2cNVVVxV+VnT/+bIvzpfeketHoiND6dOiBn1a1CgsS8/IYePeI6zbc4SU/UfZuv8489fvY9rxk52iQoJcNIiqQMOCyiC6Ao0uq0iDqIo6dITymffwyEWlp6dTtWpVIiIi2LhxIz/++KPP261bty7JycmF82c6+j+bguGS77jjjlOGS+7cuTOffPIJN998c+FwykV1796dKVOm0KtXLzZv3syuXbto2rTpKcMqb9u2jTVr1tCsWTOqVavG7bffTsWKFZk8eTJPP/30KfF7j4d/Nn379uW1115j7NixACQnJ9OmTRvS09OpU8d2Ypw8efJ574uLpUnfz1UOD6ZTg+p0anDqafRvx7PZmnaMrWnH2LL/GFvTjvNzajpzft6L94FNnSrhhWcGsdEVqFs1nJiqEcRUDScsWCsEdZL30Mrh4eHUqHHy4CMxMZHx48fTqlUrmjZtSufOnUs1tjMNl/zyyy9z++23869//YsBAwYU+ySvBx98kAceeID4+HiCgoKYPHkyoaGhTJ8+/bRhlZcvX17s0NEXGvPIkSNp1aoVubm5dO/enfHjx/PYY48xfPhwXnrpJXr16nXB279QOuBaOZOZk8eOg8dtRbD/eGGlsO3AMTJzTr1X4LLIUGKqhlO3mq0E6laNKHxfu0o4wfo84VKlA66dvxMnThAeHo6IMG3aND788EM+//xzp8O65HTANVUoLNhNs5qVaFbz1AfC5Ocb9h3NJPW3DHYfOkHqbxmk/naC3YcyWLnrN75cs5e8/JMHAC6BahVCuSwylMsq2dcalcI882GF76MjQ7VyUI5ZsWIFo0aNwhhDlSpVmDRJLx+eiyb9AOFyCbUqh1Orcjgd6lc77fPcvHx+PZLJ7kOeyuC3DPYfyWT/0Sz2H81k/Z4jHDiWRX6RE0MRqF4hhOjIMGp4Vw6eSkErB3UpdevWjdWrVzsdRpmiSV8BEOR2edr6I7C3WJwuL99w8FgW+49mse9IJvuO2Aph35Es0jyvvlQOl0WGUiUimMiwICqGFrwGUSk8iMrhwVQKC7av4fY1NMgVMF1VjTEB87eqC3OxTfKa9JXP3C6xR/CVwmhZ5/QLZgUKKgfvSqHw1XP2sOPgcY5m5nI0M4ecvLP/iEPcLiqFB1GpSIVQKSzolMqhUljwaRVHZFgQQWXkDCMsLIyDBw9SvXp1TfyqWMYYDh48SFjYhQ/mqElflTjvygHOXDkUyMrN40iGrQDSM3I4kplrXzNyOFJQlpFbOH/4RDY7Dx4vXC6v6GlFERVC3IWVQ4XQICJC3FQICSIi1L5GhgVRMSyIyFD7WjE0mIqh9uwjPMRNhGcKD3ET4r50Zx0xMTGkpqZSVgYdVM4ICwsjJibmgtfXpK8cFxrkJjrSTXRk6LkXLsIYw4nsvFMqh+IqjHTP/InsXI5m5rLvSCbHs/I4np3Lscxccs9RcRRwu4SIYFsBhIe4CQ92ExbsJjTIVfgaGuwmLMhFaLCL0CA3YUVeC5YNCXIR4rbLhbhdBAe5CAmpRlC4EOSyZUFuIdjtItgtBHleg10ufW6DumCa9FWZJiJUCA2iQmgQtSqHn3uFYhhjyMrN51iWrQCOZeVyJDOH41l5nMjOJSM7jxPZeWTk5Hm9z+WE531Wbj5ZOXkczsghK+fkfFZuPpmeV18rFV+5XWIrALeL0CAXwW4XIQWvBRWIW04pCwmyU2hhZeM+pTzY7cIt9qK/Swomez1GEDz/ISKeVzu5PGc+p5QjnteTywR7vvdkrFI4H+yZd8vJ7xYXhfMF23C7CmLSSu9CadJXAU9ECPMcsUdVPP+zDV/k5uWfUglk5uSRnZdPdq4tz87NJzsvn9w8Q05evmcy5Oblk5NvyMnNJzfflnl/np1r3xe+5uWTnWvIzssnJzefrJx8jmbm2u0XfFfh9+aRnZt/2kX3ssLlqQhcnorAu6I6WXFRpAITXGeoTMR7+VO2addze9a164mtIOXkenYbNjbvShKKVoinluG1Tr3qEYzu3fiS7jdN+kqVgiC3iyC3iwqh/ve/XK6nssjJNeQbQ56xr/n5kG8MhoLReO3yxoDBeF7tZ7bi8C7zWsbY7ZysnExhBVVQgdlKz44CmZ9vt5dfEEfB+1PK7ffm5Z98n28MeQUxe5bJK3ifz6nbOm27p28z3/M353vK8vINOXkF++jkdxZsu+DaUuH+KtjB5vQy7/1TMA922JVLzf9+gUqpUlVQIRHidCSqNJSNvmxKKaVKhCZ9pZQKIJr0lVIqgGjSV0qpAKJJXymlAogmfaWUCiCa9JVSKoBo0ldKqQCiSV8ppQKIJn2llAogmvSVUiqAaNJXSqkAoklfKaUCiCZ9pZQKIJr0lVIqgGjSV0qpAFK+kv7hXScfQaOUUuo05SfpH0iB1zvD9684HYlSSvktn5K+iCSKyCYR2SIiTxTz+QgRSRORZM90j9dneV7ls0oy+FNUbwRN+sGCv8DmeZfsa5RSqiw75zNyRcQNvA70AVKB5SIyyxizvsii040xo4rZRIYxps3Fh3oOIjDkdTi4BT75Hdz7LURd2qfKK6VUWePLkX5HYIsxZpsxJhuYBgy5tGFdoJAIuGUquEPgw1sg47DTESmllF/xJenXAXZ7zad6yoq6XkTWiMgMEanrVR4mIkki8qOIXHsxwfqkSl246T34bQd8cg/k513yr1RKqbLCl6QvxZQV7SLzBVDfGNMKWAC86/XZ5caYBGAY8LKINDztC0Tu81QMSWlpaT6Gfhb1r4D+z8OW+fDNXy9+e0opVU74kvRTAe8j9xhgj/cCxpiDxpgsz+xEoL3XZ3s8r9uARUDbol9gjJlgjEkwxiRER0ef1x9wRh1+B+3vsr15lr9VMttUSqkyzpekvxxoLCKxIhIC3AKc0gtHRGp5zQ4GNnjKq4pIqOd9FHAFUPQC8KVzzQvQJBFmj4G1n5ba1yqllL86Z+8dY0yuiIwC5gJuYJIxZp2IjAOSjDGzgNEiMhjIBQ4BIzyrNwf+KyL52Arm2WJ6/Vw67mC4cTK8fx18eh+EVYZGvUvt65VSyt+I8bM7WBMSEkxSUlLJbjTjMEweAIe2w/BZEJNQsttXSimHicgKz/XTsyo/d+SeTXgVuP0TqBgNU26AtE1OR6SUUo4IjKQPEFkT7vgMXMHw/lBIT3U6IqWUKnWBk/QBqjWA22dA1hH44Ho4ccjpiJRSqlQFVtIHqNXa3rV7aDtMvQmyjjodkVJKlZrAS/oAsd3ghrfhl5Uw9RbIPuF0REopVSoCM+kDNB8E1/0Xdn4P02+DnEynI1JKqUsucJM+QKsbYchrsPVb+HgE5GY7HZFSSl1SgZ30AdreDgP+BZu/skMy5+U6HZFSSl0ymvQBOtwD/f4JG2bBjBHa1KOUKrc06RfoMhL6/T/Y8AV8MBQyfnM6IqWUKnGa9L11eRCufxtSl8Ok/pD+i9MRKaVUidKkX1T8DXbIhiO/wNt9YP8GpyNSSqkSo0m/OLHd4a459qlbk/rBju+djkgppUqEJv0zqRkP98yHijXt0MzrZjodkVJKXTRN+mdT5XK4+2uo3db24//pv05HpJRSF0WT/rlEVIM7Z0KzAfDVYzD/acjPdzoqpZS6IJr0fREcDje9Z/vzf/8KfHa/3r2rlCqTzvm4ROXhcsM1L0Kl2vDNODi2D27+AMIqOR2ZUkr5TI/0z4cIdPsjXDveDtT2zjVweLfTUSmllM806V+INrfCsI/g8E6Y2BN2/eh0REop5RNN+heqUW+4ZwGERsLkgbDyfacjUkqpc9KkfzGim8K930L9K2HWKPjqcR2lUynl1zTpX6zwqnDbDOg8En4aD+9fC8f2Ox2VUkoVS5N+SXAHQeI/7QXe1CQY3w12/uB0VEopdRpN+iWpza1w7zcQUsG283//ChjjdFRKKVVIk35JqxEH9y2C5gPt3bvTboOMw05HpZRSgCb9SyOsEtz4LiQ+BylzYUIP2JPsdFRKKaVJ/5IRgc4PwF1fQV4OvN0Xkt7R5h6llKM06V9qdTvC/Yttt84vH7Hj9mQdczoqpVSA0qRfGipUt906e/4Zfv7Y3sWrT+RSSjlAk35pcbmgx1i4Y6a9sDuhJyRPdToqpVSA0aRf2hr0gAeWQEwCzPw9fD4Ssk84HZVSKkD4lPRFJFFENonIFhF5opjPR4hImogke6Z7vD4bLiIpnml4SQZfZkXWsEf83cfCqg/grashbbPTUSmlAsA5k76IuIHXgf5AC+BWEWlRzKLTjTFtPNNbnnWrAc8AnYCOwDMiUrXEoi/L3EHQ689w2ydwdC/8tzssf1t79yilLilfjvQ7AluMMduMMdnANGCIj9vvB8w3xhwyxvwGzAcSLyzUcqrx1fD7H6BeF5j9B5hyI/y20+molFLllC9Jvw7g/aSQVE9ZUdeLyBoRmSEidc9nXRG5T0SSRCQpLS3Nx9DLkUq17BF/4nN2zJ7XO8GSf+uInUqpEudL0pdiyoq2QXwB1DfGtAIWAO+ex7oYYyYYYxKMMQnR0dE+hFQOuVz2Zq5Ry+xY/Qv+ApOvgUPbnY5MKVWO+JL0U4G6XvMxwB7vBYwxB40xWZ7ZiUB7X9dVRVSOgVumwPVvw/6NdsTOpHcgP9/pyJRS5YAvSX850FhEYkUkBLgFmOW9gIjU8podDBTceTQX6CsiVT0XcPt6ytS5xN8Av18CtdvYO3nf6W8rAaWUugjnTPrGmFxgFDZZbwA+MsasE5FxIjLYs9hoEVknIquB0cAIz7qHgL9hK47lwDhPmfJFlcth+Bcw5HU4sAnGXwnf/h1yMp2OTClVRonxsy6CCQkJJikpyekw/M/xAzD3T7BmOlRrCINehtjuTkellPITIrLCGJNwruX0jtyyokIUDJ0Ad3wGJg/eHQQzH4QTeuKklPKdJv2ypmEv+P1SuPJRe9T/WgKsnqY3dSmlfKJJvywKiYCr/wL3fwfVGtjhmt+/Fg5udToypZSf06RfltWIg7vnwTUvwi8r4c2u8N2LkJvtdGRKKT+lSb+sc7mg470wchk07gvf/s0m/60LnY5MKeWHNOmXF5Vqwc3v24e15Ofa5p6PR8ARvRdOKXWSJv3ypnEfePBHuOpPsHEOvNYBfviPfU6vUirgadIvj4LD4KrHYeRPUO8KmPdnO5zDjiVOR6aUcpgm/fKsWiwMmw63fAjZx2HyAJhxN6SnOh2ZUsohmvTLOxFodo096u/xBGycbZt8/veCDuegVADSpB8oQiKg55O2l0+jq2Hh3+H1jrDhS72xS6kAokk/0FStZ3v53DkLgiNg+m3w/nWQtsnpyJRSpUCTfqBq0AMeWAL9n4c9nhu7vv4TZKY7HZlMVQ98AAASI0lEQVRS6hLSpB/I3EHQ6X54aCW0vR1+fANebQc/vgm5WedeXylV5mjSV3YEz0GvwH2L4LLm8PUT8J/2sPJ9fU6vUuWMJn11Uu029qEtd8yECtEwaxS80RnWfqqPa1SqnNCkr04lAg17wr3fws0fgMsNM+6yT+3a8IX29FGqjNOkr4onAs0Hwe9/gKFvQW4mTL8d/tsdNn2tyV+pMkqTvjo7lxta3Wj791/7JmQdgQ9vhom9IGWBJn+lyhhN+so37iBoMwxGJcHg/9hn9k65Ht7ua4dx1uSvVJmgSV+dH3cwtLsTHloBA/8NR36xwzi/cw1sX+x0dEqpc9Ckry5MUAgk3A2jV9kndx3aBu8OhMkDYedSp6NTSp2BJn11cYJC7ZO7Hk6GxGftcA7vJNqhHXYvdzo6pVQRmvRVyQgOh86/h4dXQ9+/w97V8PbVMOVG2LPK6eiUUh6a9FXJComArg/Bw2ug9zOQuhwmXAWf3AOHdzkdnVIBT5O+ujRCK0K3P9jk3+2P9sau/7SHz0fCvvVOR6dUwNKkry6tsErQ+2nb26ftHfDzJ/BmF9vmv0X7+StV2jTpq9JROQYGvgR/WA+9/s8e7X9wPbzRBVa+p0/xUqqUaNJXpSuiGnQfA4/8DNf9F1xBMOsh+HcczPs/OLDF6QiVKtfE+NnpdUJCgklKSnI6DFVajIEdi+Gn/8Kmr8DkQb0rof1waD4YgsOcjlCpMkFEVhhjEs65nCZ95TeO7oPkKba557ftEFYFWt8C7e+Cy5o5HZ1Sfs3XpO9T846IJIrIJhHZIiJPnGW5G0TEiEiCZ76+iGSISLJnGu/7n6ACTmQN2+PnoZX2Gb6NekPSJHijE0y7DfYkOx2hUmVe0LkWEBE38DrQB0gFlovILGPM+iLLRQKjgZ+KbGKrMaZNCcWrAoHLZZ/h26AHHD8IyyfaRzlu/BKaJEL3xyCmvdNRKlUm+XKk3xHYYozZZozJBqYBQ4pZ7m/A84B2w1Alp0J1uOoJe+G3159h90/wVi94dxBsnAP5eU5HqFSZ4kvSrwPs9ppP9ZQVEpG2QF1jzJfFrB8rIqtE5H8i0u3CQ1UBLawydB9rk//Vf4WDW2HarfCfdrD0dchMdzpCpcoEX5K+FFNWePVXRFzAv4E/FrPcXuByY0xb4A/AVBGpdNoXiNwnIkkikpSWluZb5CowhUbClY/YMX5unAwVa8LcP8G/msPsMXAgxekIlfJrviT9VKCu13wMsMdrPhJoCSwSkR1AZ2CWiCQYY7KMMQcBjDErgK1Ak6JfYIyZYIxJMMYkREdHX9hfogKLOxjiroPfzYX7FkGLwbDyXXgtwd70lTJfH+auVDF8SfrLgcYiEisiIcAtwKyCD40x6caYKGNMfWNMfeBHYLAxJklEoj0XghGRBkBjYFuJ/xUqsNVuC9eNh0fXwVV/gl9/hik3wOsd4KcJkHXU6QiV8hvnTPrGmFxgFDAX2AB8ZIxZJyLjRGTwOVbvDqwRkdXADOABY8yhiw1aqWJVvAyuehweWWsf5h5WGb4aCy+1gK+ftA96USrA6c1ZqnxLTYIf34T1M21Pn8Z97BO/Gve1D31XqpzQO3KV8nZkr73Ra+V7cOxXqBRjn/Xb7g6oVNvp6JS6aJr0lSpOXo4d42fFO7D1WxA3NO1vj/4b9LQ3hilVBvma9M95R65S5Yo72Pb0aTHY9vVf+S6s+sDe7Vu1vh3np+3tUCHK6UiVuiT0SF+p3Cz7ZK+kSbDze3CH2BE+24+Aelfo0b8qE/RIXylfBYVC/A122r8RVkyG1VNh7QyoUs+O9Nn6FqjWwOlIlbpoeqSvVHGyT9gmn+SpsG0RYODyLjb5x11nu4Mq5Uf0Qq5SJSX9F1gzHVZ/CAc2Q1AYNBsArW+1F3/desKsnKdJX6mSZgzsWQnJH9qmn4zfoGINaHWTrQBqxDkdoQpgmvSVupRys2DzXFg9DVLmQn4u1GwFbYZB/I3a+0eVOk36SpWW4wfg5xn24u/e1bb3T4shkPA7uLwzSHED1SpVsjTpK+WEfett3//kqZB1xPb4aXWznarFOh2dKsc06SvlpOzjsG4mrJkG2xcDBup29vT+uRbCqzodoSpnNOkr5S/SU2HNR7b9/8Am2/zTtL8d+6dBTx34TZUITfpK+RtjYG8yrJ5uu4BmHILKdaHpNdCoN9TvBiERTkepyihN+kr5s9ws2Djb9v3fvhhyMyAk0jb9tLlNLwCr86bDMCjlz4JCoeVQO+Vk2jF/1n4Caz+FVe9D1Vjb9z/+Bqje0OloVTmiR/pK+ZOsY3bwt9VTYft3tqxWG2h5vR3+oUrds6+vApY27yhV1qWn2h5Aaz+xdwIDxHS0Q0A0GwhRjZyNT/kVTfpKlSeHttmmnw2z7A1gYM8AWt1szwIiazgbn3KcJn2lyqvDu23yX/OR7Q0kLtv1s+VQaNRHK4AApUlfqUCQtskm/zUfQfouW1a7nR3/J/4GqHiZs/GpUqNJX6lAYgz8ugZS5tkLwXtX2+f/Nuptm4CaDYDgcKejVJeQJn2lAtn+DfYGsDUfwZFf7D0ALYbYYSD0EZDlkiZ9pRTk58POJfYu4PWfQ/ZRexdw/I22EqjVWm8CKyc06SulTpV9AjbNsWMAbf0GTD5UirHjADW7BupdCUEhTkepLpAmfaXUmR0/AJu/ho1zYOu3dhiI0ErQ6Grb/t/oagiv4nSU6jzoMAxKqTOrEAVtb7dTToZ9+PvG2bYiWPcpuIKg/pXQdIA9E9A7gcsNPdJXSp2UnwepSbBpNmz6yj4IHuyjIJsPhuaDILqpXgfwQ9q8o5S6eAe22Apg42zY/ZMtq97IJv/mg6F2W60A/IQmfaVUyTqy11YAG76EHYvtw+Cr1LPDQcddZ4eF0ArAMZr0lVKXzolD9uh//Ux7PUArAMdp0ldKlQ6tAPyCr0nfp9vyRCRRRDaJyBYReeIsy90gIkZEErzKnvSst0lE+vkWvlKqzIioBu3ugNs/gTEpMPg1iGoMS1+HCVfBK61h/tOwZ5UdLkI56pxH+iLiBjYDfYBUYDlwqzFmfZHlIoHZQAgwyhiTJCItgA+BjkBtYAHQxBiTd6bv0yN9pcqJM50BtBgMzYdAnfY6HEQJKsl++h2BLcaYbZ4NTwOGAOuLLPc34HlgjFfZEGCaMSYL2C4iWzzbW+rD9yqlyrKCM4B2d5xaAfw4Hn74D0TWhuYDbS+gel3B5XY64oDgS9KvA+z2mk8FOnkvICJtgbrGmC9FZEyRdX8ssm6dol8gIvcB9wFcfvnlvkWulCo7vCuAjMOwea59JsDK92DZBIiIskNBNB8Csd11OIhLyJekX9wVmMI2IRFxAf8GRpzvuoUFxkwAJoBt3vEhJqVUWRVeBVrfbKesY7Blvh0Oeu2nthIIrWwvAifcZe8DUCXKl6SfCnjfgx0D7PGajwRaAovEXqGvCcwSkcE+rKuUCmShFW0Pn7jrICcTti20zwVe8xGsfBdqtIQW19rrANFNnY62XPDlQm4Q9kJub+AX7IXcYcaYdWdYfhEwxnMhNw6YyskLud8AjfVCrlLqrDIOw88fw88zYLenhTi62cmhIGrGazfQIkrsQq4xJldERgFzATcwyRizTkTGAUnGmFlnWXediHyEveibC4w8W8JXSinANgF1vNdOR/bCxi/t8wAWvwjfPQ9VLodmA+2IoJd30YvA50FvzlJKlR3H0mDzV3YoiG2LIC8LIqpDk/62J1CDnhAc5nSUjtA7cpVS5VvWUdiywDMk9DzISrfPBGg20FYA9btBWCWnoyw1Op6+Uqp8C408eRE4Nxu2f2efBbDhC1g91T4Yvm5Hex2gxWCoHON0xH5Bj/SVUuVLbhbsXmabfzZ/DfvW2vKYDva5wC2G2GsC5Yw27yilFNhnAmz43HYF/XWNLavdzl4Eju1u7wVwBzsbYwnQpK+UUkUd2gbrZ9nhIPassmUhFW0TUJthUO+KMjsekCZ9pZQ6m+MHYOf3kDIP1n0O2UftcBCN+9ipYS8Ir+p0lD7TpK+UUr7KPgGb5thrAFsWQMZvngvBnWwF0KQfXNbCr28I06SvlFIXouDh8Cnz7FRwHaBSjOcsoK+9FhBa0dk4i9Ckr5RSJeHIXjsoXMo82LrINgO5Q6D+lbYCaNwXqjd0OkpN+kopVeJys2HX0pNnAQc22/JqDT0VQB9bGQSFlnpomvSVUupSO7TdXgPYPBd2LIbcTAiuAA172tFBmybam8hKgSZ9pZQqTdknYMcSSJkLG+fA0T3gDoXLO9lrALFXee4JuDQDIWjSV0opp+Tnw+6f7JAQ2/938q7gkEio18U2AdXvBrXalNh9ATr2jlJKOcXlssm9Xhc7f/yAbf7Z9j/P2cA8W14hGhr3s11CG/YslaYgTfpKKXWpVYg6OTgcwNF99gxg81zY+AUkfwCuYDs66I2TL2komvSVUqq0RdaAVjfZKS/XNgVt/hpclz4la9JXSiknuYOg/hV2KgVlc2QhpZRSF0STvlJKBRBN+kopFUA06SulVADRpK+UUgFEk75SSgUQTfpKKRVANOkrpVQA8bsB10QkDdjpdBxnEAUccDqIC6Bxly6Nu3Rp3FY9Y0z0uRbyu6Tvz0QkyZdR7PyNxl26NO7SpXGfH23eUUqpAKJJXymlAogm/fMzwekALpDGXbo07tKlcZ8HbdNXSqkAokf6SikVQDTpn4WIuEVklYh86ZmPFZGfRCRFRKaLSIjTMRYlIlVEZIaIbBSRDSLSRUSqich8T9zzRaSq03EWJSKPisg6EVkrIh+KSJi/7m8RmSQi+0VkrVdZsftYrFdFZIuIrBGRdn4W9wue38oaEflMRKp4ffakJ+5NItLPmaiLj9vrszEiYkQkyjPv1/vbU/6QZ5+uE5HnvcpLZX9r0j+7h4ENXvPPAf82xjQGfgN+50hUZ/cK8LUxphnQGhv/E8A3nri/8cz7DRGpA4wGEowxLQE3cAv+u78nA4lFys60j/sDjT3TfcCbpRRjcSZzetzzgZbGmFbAZuBJABFpgf03iPOs84aIuEsv1FNM5vS4EZG6QB9gl1exX+9vEekJDAFaGWPigBc95aW2vzXpn4GIxAADgLc88wL0AmZ4FnkXuNaZ6IonIpWA7sDbAMaYbGPMYeyP7F3PYn4Xt0cQEC4iQUAEsBc/3d/GmO+AQ0WKz7SPhwDvGetHoIqI1CqdSE9VXNzGmHnGmFzP7I9AjOf9EGCaMSbLGLMd2AJ0LLVgT42xuP0N8G/gMcD7wqRf72/g98CzxpgszzL7PeWltr816Z/Zy9gfVL5nvjpw2Ot/kFSgjhOBnUUDIA14x9Ms9ZaIVABqGGP2AnheL3MyyKKMMb9gj3h2YZN9OrAC/9/f3s60j+sAu72W8+e/427gK897v45bRAYDvxhjVhf5yK/jBpoA3TzNlv8TkQ6e8lKLW5N+MURkILDfGLPCu7iYRf2t61MQ0A540xjTFjiOnzXlFMfT/j0EiAVqAxWwp+lF+dv+9kVZ+N0gIk8BucCUgqJiFvOLuEUkAngKeLq4j4sp84u4PYKAqkBnYCzwkacVodTi1qRfvCuAwSKyA5iGbWZ4GXuqWPAw+RhgjzPhnVEqkGqM+ckzPwNbCewrOMX1vO4/w/pOuRrYboxJM8bkAJ8CXfH//e3tTPs4FajrtZzf/R0iMhwYCNxmTvbh9ue4G2IPEFZ7/h+NAVaKSE38O26w8X3qaX5ahm1JiKIU49akXwxjzJPGmBhjTH3sxZVvjTG3AQuBGzyLDQc+dyjEYhljfgV2i0hTT1FvYD0wCxsv+GHc2GadziIS4TnqKYjbr/d3EWfax7OAOz29SjoD6QXNQP5ARBKBx4HBxpgTXh/NAm4RkVARicVeGF3mRIxFGWN+NsZcZoyp7/l/NBVo5/n9+/X+BmZiDyIRkSZACHbQtdLb38YYnc4yAVcBX3reN/D8Q2wBPgZCnY6vmHjbAEnAGs8PrCr2esQ3QIrntZrTcRYT91+BjcBa4H0g1F/3N/Ah9tpDDjbh/O5M+xh72v46sBX4GdtDyZ/i3oJtS072TOO9ln/KE/cmoL8/xV3k8x1AVBnZ3yHAB57f+UqgV2nvb70jVymlAog27yilVADRpK+UUgFEk75SSgUQTfpKKRVANOkrpVQA0aSvlFIBRJO+UkoFEE36SikVQP4/JMG91oZixI8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bca2c7bcc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def modelfit(alg, x, y, cv_folds=10, early_stopping_rounds=50):\n",
    "   \n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgtrain = xgb.DMatrix(x, label = y)\n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], nfold=cv_folds,\n",
    "        metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "    alg.set_params(n_estimators=cvresult.shape[0])\n",
    "    alg.fit(x,y,eval_metric='mlogloss')\n",
    "    cvresult.loc[30:,[\"test-mlogloss-mean\", \"train-mlogloss-mean\"]].plot()\n",
    "      \n",
    "    print('n_estimators: ',cvresult.shape[0])\n",
    "    train_predprob = alg.predict_proba(x)\n",
    "    logloss = log_loss(y, train_predprob)\n",
    "\n",
    "    print(logloss)\n",
    "    \n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=3000,\n",
    "        max_depth=8,\n",
    "        num_class=3,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        nthread=4,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.8,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=20)\n",
    "modelfit(xgb1, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在确定n_estimators数量后利用GridSearchCV 对 max_depth 和 min_child_weight 进行自动调参"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=163,\n",
       "       n_jobs=1, nthread=4, num_class=3, objective='multi:softprob',\n",
       "       random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "       seed=20, silent=True, subsample=0.8),\n",
       "       fit_params=None, iid=False, n_jobs=1,\n",
       "       param_grid={'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 10, 2)},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {\n",
    "    'max_depth':range(3,10,2),\n",
    "    'min_child_weight':range(1,10,2)\n",
    "}\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=163, max_depth=5,num_class=3,nthread=4,\n",
    "                                        min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    "                                        objective= 'multi:softprob',  seed=20), \n",
    "                       param_grid = param_test1, scoring='neg_log_loss',iid=False, cv=5)\n",
    "gsearch1.fit( X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: -0.60958, std: 0.00446, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.60950, std: 0.00468, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.60936, std: 0.00462, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.60922, std: 0.00444, params: {'max_depth': 3, 'min_child_weight': 7},\n",
       "  mean: -0.60929, std: 0.00452, params: {'max_depth': 3, 'min_child_weight': 9},\n",
       "  mean: -0.59068, std: 0.00437, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.59084, std: 0.00391, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.59095, std: 0.00404, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.59070, std: 0.00392, params: {'max_depth': 5, 'min_child_weight': 7},\n",
       "  mean: -0.59060, std: 0.00465, params: {'max_depth': 5, 'min_child_weight': 9},\n",
       "  mean: -0.58505, std: 0.00526, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.58468, std: 0.00454, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58484, std: 0.00465, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.58438, std: 0.00373, params: {'max_depth': 7, 'min_child_weight': 7},\n",
       "  mean: -0.58417, std: 0.00438, params: {'max_depth': 7, 'min_child_weight': 9},\n",
       "  mean: -0.59125, std: 0.00350, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.58737, std: 0.00468, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.58734, std: 0.00436, params: {'max_depth': 9, 'min_child_weight': 5},\n",
       "  mean: -0.58587, std: 0.00459, params: {'max_depth': 9, 'min_child_weight': 7},\n",
       "  mean: -0.58532, std: 0.00422, params: {'max_depth': 9, 'min_child_weight': 9}],\n",
       " {'max_depth': 7, 'min_child_weight': 9})"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=2, missing=None, n_estimators=163,\n",
       "       n_jobs=1, nthread=4, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=27, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params=None, iid=False, n_jobs=4,\n",
       "       param_grid={'max_depth': [4, 5, 6], 'min_child_weight': [1, 5, 9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 范围缩小调优\n",
    "param_test2 = {\n",
    "    'max_depth':[4,5,6],\n",
    "    'min_child_weight':[1,5,9]\n",
    "}\n",
    "gsearch2 = GridSearchCV(estimator = XGBClassifier( learning_rate=0.1, n_estimators=163, max_depth=5,\n",
    "                                        min_child_weight=2, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    "                                        objective= 'multi:softprob', nthread=4, scale_pos_weight=1,seed=27), \n",
    "                       param_grid = param_test2, scoring='neg_log_loss',n_jobs=4,iid=False, cv=5)\n",
    "gsearch2.fit( X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: -0.59807, std: 0.00421, params: {'max_depth': 4, 'min_child_weight': 1},\n",
       "  mean: -0.59852, std: 0.00408, params: {'max_depth': 4, 'min_child_weight': 5},\n",
       "  mean: -0.59813, std: 0.00406, params: {'max_depth': 4, 'min_child_weight': 9},\n",
       "  mean: -0.59101, std: 0.00434, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.59076, std: 0.00391, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.59110, std: 0.00423, params: {'max_depth': 5, 'min_child_weight': 9},\n",
       "  mean: -0.58616, std: 0.00469, params: {'max_depth': 6, 'min_child_weight': 1},\n",
       "  mean: -0.58647, std: 0.00396, params: {'max_depth': 6, 'min_child_weight': 5},\n",
       "  mean: -0.58648, std: 0.00404, params: {'max_depth': 6, 'min_child_weight': 9}],\n",
       " {'max_depth': 6, 'min_child_weight': 1},\n",
       " -0.58615716498905113)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2.grid_scores_, gsearch2.best_params_, gsearch2.best_score_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=6, min_child_weight=2, missing=None, n_estimators=163,\n",
       "       n_jobs=1, nthread=4, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=27, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params=None, iid=False, n_jobs=4,\n",
       "       param_grid={'min_child_weight': [1, 6, 8, 10, 12]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 单独对 min_child_weight 调优\n",
    "param_test2b = {\n",
    "    'min_child_weight':[1,6,8,10,12]\n",
    "}\n",
    "gsearch2b = GridSearchCV(estimator = XGBClassifier( learning_rate=0.1, n_estimators=163, max_depth=6,\n",
    "                                        min_child_weight=2, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    "                                        objective= 'multi:softprob', nthread=4, scale_pos_weight=1,seed=27), \n",
    "                       param_grid = param_test2b, scoring='neg_log_loss',n_jobs=4,iid=False, cv=5)\n",
    "gsearch2b.fit( X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: -0.58616, std: 0.00469, params: {'min_child_weight': 1},\n",
       "  mean: -0.58653, std: 0.00398, params: {'min_child_weight': 6},\n",
       "  mean: -0.58630, std: 0.00373, params: {'min_child_weight': 8},\n",
       "  mean: -0.58650, std: 0.00407, params: {'min_child_weight': 10},\n",
       "  mean: -0.58626, std: 0.00382, params: {'min_child_weight': 12}],\n",
       " {'min_child_weight': 1},\n",
       " -0.58615716498905113)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2b.grid_scores_, gsearch2b.best_params_, gsearch2b.best_score_\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对 gamma 进行调参"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=6, min_child_weight=1, missing=None, n_estimators=163,\n",
       "       n_jobs=1, nthread=4, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=27, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params=None, iid=False, n_jobs=4,\n",
       "       param_grid={'gamma': [0.0, 0.1, 0.2, 0.3, 0.4]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test3 = {\n",
    "    'gamma':[i/10.0 for i in range(0,5)]\n",
    "}\n",
    "gsearch3 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=163, max_depth=6,\n",
    "                                        min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    "                                        objective= 'multi:softprob', nthread=4, scale_pos_weight=1,seed=27), \n",
    "                       param_grid = param_test3, scoring='neg_log_loss',n_jobs=4,iid=False, cv=5)\n",
    "gsearch3.fit( X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: -0.58616, std: 0.00469, params: {'gamma': 0.0},\n",
       "  mean: -0.58640, std: 0.00436, params: {'gamma': 0.1},\n",
       "  mean: -0.58607, std: 0.00407, params: {'gamma': 0.2},\n",
       "  mean: -0.58653, std: 0.00396, params: {'gamma': 0.3},\n",
       "  mean: -0.58637, std: 0.00434, params: {'gamma': 0.4}],\n",
       " {'gamma': 0.2},\n",
       " -0.58606930251231015)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3.grid_scores_, gsearch3.best_params_, gsearch3.best_score_\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对正则参数 reg_alpha 和 reg_lambda 调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.8, gamma=0.2, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=6, min_child_weight=1, missing=None,\n",
       "       n_estimators=163, n_jobs=1, nthread=4, objective='multi:softprob',\n",
       "       random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "       seed=27, silent=True, subsample=0.8),\n",
       "       fit_params=None, iid=False, n_jobs=4,\n",
       "       param_grid={'reg_alpha': [1e-05, 0.01, 0.1, 1, 100], 'reg_lambda': [2, 3, 4]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test4 = {  \n",
    " 'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100],\n",
    "#   'reg_lambda': [1, 2, 5, 7]\n",
    " 'reg_lambda': [2, 3, 4]\n",
    "}  \n",
    "gsearch4 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=163, max_depth=6, min_child_weight=1, gamma=0.2,\n",
    "                                                  subsample=0.8, colsample_bytree=0.8, objective= 'multi:softprob', nthread=4, \n",
    "                                                  scale_pos_weight=1,seed=27), param_grid = param_test4, scoring='neg_log_loss',n_jobs=4,iid=False, cv=5)  \n",
    "  \n",
    "gsearch4.fit( X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: -0.58634, std: 0.00392, params: {'reg_alpha': 1e-05, 'reg_lambda': 2},\n",
       "  mean: -0.58676, std: 0.00385, params: {'reg_alpha': 1e-05, 'reg_lambda': 3},\n",
       "  mean: -0.58649, std: 0.00366, params: {'reg_alpha': 1e-05, 'reg_lambda': 4},\n",
       "  mean: -0.58603, std: 0.00419, params: {'reg_alpha': 0.01, 'reg_lambda': 2},\n",
       "  mean: -0.58670, std: 0.00381, params: {'reg_alpha': 0.01, 'reg_lambda': 3},\n",
       "  mean: -0.58651, std: 0.00407, params: {'reg_alpha': 0.01, 'reg_lambda': 4},\n",
       "  mean: -0.58670, std: 0.00387, params: {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  mean: -0.58669, std: 0.00385, params: {'reg_alpha': 0.1, 'reg_lambda': 3},\n",
       "  mean: -0.58657, std: 0.00407, params: {'reg_alpha': 0.1, 'reg_lambda': 4},\n",
       "  mean: -0.58625, std: 0.00413, params: {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  mean: -0.58635, std: 0.00412, params: {'reg_alpha': 1, 'reg_lambda': 3},\n",
       "  mean: -0.58640, std: 0.00399, params: {'reg_alpha': 1, 'reg_lambda': 4},\n",
       "  mean: -0.62240, std: 0.00439, params: {'reg_alpha': 100, 'reg_lambda': 2},\n",
       "  mean: -0.62256, std: 0.00438, params: {'reg_alpha': 100, 'reg_lambda': 3},\n",
       "  mean: -0.62234, std: 0.00445, params: {'reg_alpha': 100, 'reg_lambda': 4}],\n",
       " {'reg_alpha': 0.01, 'reg_lambda': 2},\n",
       " -0.58603102752254599)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch4.grid_scores_, gsearch4.best_params_, gsearch4.best_score_  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 在前面选择的参数下 再次训练模型 以选出最优 n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators:  317\n",
      "0.435764923067\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8FEXawPFfzWRyhyuJ3JCAIJCEBEgAQUFEOUQRURRRAW8ERN1VV991vd31ZNFVUVHAAwVFARV3BRQEFAUiyH0TIIAQCAmBnDNT7x/dCSEHDOToTOb5fj7jzPT5dAaf6q6urlJaa4QQQvgGm9UBCCGEqD6S9IUQwodI0hdCCB8iSV8IIXyIJH0hhPAhkvSFEMKHSNIXQggfIklfCCF8iCR9IYTwIX5WB1BSRESEjoqKsjoMIYTwKsnJyUe01pFnW67GJf2oqChWr15tdRhCCOFVlFJ7PFlOqneEEMKHSNIXQggfIklfCCF8SI2r0xfCVxUUFJCamkpubq7VoYgaLDAwkGbNmuFwOM5rfUn6QtQQqamphIWFERUVhVLK6nBEDaS15ujRo6SmphIdHX1e25DqHSFqiNzcXMLDwyXhi3IppQgPD6/Q1aAkfSFqEEn44mwq+m+k1iT9k3lOXv7fFvYcPWl1KEIIUWPVmqR/Is/J9F9SeGH+ZqtDEcIrZWRk8Pbbb5/XupMmTSI7O7tC+x89ejSzZ88+r3WnT5/O+PHjK7R/X1Frkn7DOoGM63MhCzYdYvn2I1aHI4TXsTrpi+pRa5I+wJ2XRNOiQTDPfLMRp8ttdThCeJXHHnuMnTt3kpCQwCOPPMIrr7xCUlISHTt25KmnngLg5MmTDBo0iPj4eGJjY5k1axZvvPEGBw4coE+fPvTp06fUdqdPn86QIUO45ppriI6O5s0332TixIl06tSJ7t27k56eXmqdH374gU6dOhEXF8cdd9xBXl4eAN999x3t2rXjkksuYcKECVx99dWl1t2zZw99+/alY8eO9O3bl7179wLwxRdfEBsbS3x8PL169QJg48aNdO3alYSEBDp27Mj27dtLbe/pp59m1KhR9OvXj6ioKL766iseffRR4uLiGDBgAAUFBQAkJyfTu3dvunTpQv/+/Tl48CAAU6ZMISkpifj4eK6//vqiwnH06NFMmDCBHj160KpVq/O+yjlXtarJZqDDzt8Htefej5P5aMUe7rjk/Jo0CWG1Z77ZyKYDxyt1mx2a1OGpa2LKnf/iiy+yYcMG1q5dy4IFC5g9ezYrV65Ea83gwYNZunQpaWlpNGnShPnz5wOQmZlJ3bp1mThxIosXLyYiIqLMbW/YsIE1a9aQm5vLhRdeyEsvvcSaNWt46KGH+Oijj3jwwQeLls3NzWX06NH88MMPtG3blpEjRzJ58mTGjBnDvffey9KlS4mOjubmm28uc1/jx49n5MiRjBo1iqlTpzJhwgTmzp3Ls88+y/fff0/Tpk3JyMgA4J133uGBBx7glltuIT8/H5fLVeY2d+7cyeLFi9m0aRMXX3wxX375JS+//DLXXXcd8+fPZ9CgQdx///3MmzePyMhIZs2axd///nemTp3K0KFDufvuuwF44okn+OCDD7j//vsBOHjwIMuXL2fLli0MHjyYG2644Sy/YsXVqjN9gH4dGtK7bSSvLtjKvnS53BTifCxYsIAFCxbQqVMnOnfuzJYtW9i+fTtxcXEsWrSIv/3tbyxbtoy6det6tL0+ffoQFhZGZGQkdevW5ZprrgEgLi6OlJSU05bdunUr0dHRtG3bFoBRo0axdOlStmzZQqtWrYrap5eX9FesWMGIESMAuO2221i+fDkAPXv2ZPTo0UyZMqUouV988cX885//5KWXXmLPnj0EBQWVuc2BAwficDiIi4vD5XIxYMCA0+LfunUrGzZs4MorryQhIYHnn3+e1NRUwCjwLr30UuLi4pgxYwYbN24s2u6QIUOw2Wx06NCBQ4cOefS3rKhadaYPRnOmF66Lpf+/l/J/c9bz0R1dpRmc8DpnOiOvDlprHn/8ce69995S85KTk/nuu+94/PHH6devH08++eRp8+fMmcMzzzwDwPvvvw9AQEBA0XybzVb03Waz4XQ6S+27vJjOR+H//++88w6//fYb8+fPJyEhgbVr1zJixAi6devG/Pnz6d+/P++//z6bN29mypQpgFGdVDx+m82Gw+Eo2mZh/FprYmJiWLFiRan9jx49mrlz5xIfH8/06dNZsmRJ0bzif5fzPb5zVevO9AGa1Q/mbwPbsWz7EWYnp1odjhBeISwsjKysLAD69+/P1KlTOXHiBAD79+/n8OHDHDhwgODgYG699VYefvhhfv/991LrXnfddaxdu5a1a9eSmJh4znG0a9eOlJQUduzYAcDHH39M7969adeuHbt27Sq6Mpg1a1aZ6/fo0YOZM2cCMGPGDC655BLAqKLp1q0bzz77LBEREezbt49du3bRqlUrJkyYwODBg1m3bh3jxo0rir9JkyYexXzRRReRlpZWlPQLCgqKzuizsrJo3LgxBQUFzJgx45z/HpWt1p3pF7q1W0u+XXeQZ77ZRFJUA6IiQqwOSYgaLTw8nJ49exIbG8vAgQMZMWIEF198MQChoaF88skn7Nixg0ceeaTojHfy5MkA3HPPPQwcOJDGjRuzePHiCsURGBjItGnTGDZsGE6nk6SkJMaMGUNAQABvv/02AwYMICIigq5du5a5/htvvMEdd9zBK6+8QmRkJNOmTQPgkUceYfv27Wit6du3L/Hx8bz44ot88sknOBwOGjVqVOqqxVP+/v7Mnj2bCRMmkJmZidPp5MEHHyQmJobnnnuObt260bJlS+Li4ooKR6uo6rqk8FRiYqKurEFU9mfkcNXry2gZHszsMT3w96uVFzailti8eTPt27e3Oowa7cSJE4SGhqK1Zty4cbRp04aHHnrI6rCqXVn/VpRSyVrrs15a1eos2LReEC9d35F1qZm88v0Wq8MRQlTQlClTSEhIICYmhszMzDLvOYgzq7XVO4UGxDbi1u4tmLJsNz0ujKDPRRdYHZIQ4jw99NBDPnlmX5lq9Zl+oScGdaBdozAenLmWbYesrU8TQggr+UTSD3TYee+2RPz9bNz2wW/Sfl8I4bN8IukDtAgP5qM7upKT72Lk1JUcOZFndUhCCFHtfCbpA7RvXIepo5M4mJnDqKkrycotsDokIYSoVj6V9AESoxow+ZYubP0zi7s+XE1Oftl9bQjha863l82rrrqqqC+bylSR7pJTUlKIjY2t5IhqB59L+gB92l3AazfGsyolXc74hTCVl/TL64Ss0HfffUe9evWqKixRyXwy6QNcm9CU14d34ve9x7j1/d9IP5lvdUhCWKp418pJSUn06dOHESNGEBcXBxidg3Xp0oWYmBjee++9ovWioqI4cuQIKSkptG/fnrvvvpuYmBj69etHTk5Oqf2kpKTQrl077rrrLmJjY7nllltYtGgRPXv2pE2bNqxcubLUOuV1l7xz5066d+9OUlISTz75JKGhoaXWzc3N5fbbbycuLo5OnToVPTFcVrfKZXUdXdKSJUvo3bs3N954I23btuWxxx5jxowZdO3albi4OHbu3AlAWloa119/PUlJSSQlJfHzzz8DsHLlSnr06EGnTp3o0aMHW7duBYwrm6FDhzJgwADatGnDo48+ek6/n8e01jXq1aVLF12dFmz8U7f5+3e618s/6h2Hs6p130IUt2nTplNfvvub1lOvqtzXd3874/53796tY2JitNZaL168WAcHB+tdu3YVzT969KjWWuvs7GwdExOjjxw5orXWumXLljotLU3v3r1b2+12vWbNGq211sOGDdMff/xxmfux2+163bp12uVy6c6dO+vbb79du91uPXfuXH3ttddqrbWeNm2aHjdunNZa66uvvlpPnz5da631Bx98ULTMoEGD9Keffqq11nry5Mk6JCSk1LG8+uqrevTo0VprrTdv3qybN2+uc3Jy9Pjx4/Unn3yitdY6Ly9PZ2dn69mzZ+u77rqrKNaMjIxS8S9evFjXrVtXHzhwQOfm5uomTZroJ598Umut9aRJk/QDDzygtdb65ptv1suWLdNaa71nzx7drl07rbXWmZmZuqCgQGut9cKFC/XQoUOLjjc6OlpnZGTonJwc3aJFC713794yf6vT/q2YgNXagxzrs2f6ha7s0JDP7u7OiVwnQ9/+haXb0qwOSYgaoWvXrkXdGIPRp018fDzdu3dn3759ZQ44Eh0dTUJCAgBdunQp1W1y8eXi4uKw2WzExMTQt29flFJldrUM5XeXvGLFCoYNGwZQNL+k5cuXc9tttwFGZ24tW7Zk27ZtZXar7GnX0UlJSTRu3JiAgABat25Nv379gNO7il60aBHjx48nISGBwYMHc/z4cbKyssjMzGTYsGHExsby0EMPndbVct++falbty6BgYF06NCBPXv2lLn/iqj1T+R6okvL+swZ25O7PlrFqGkrGXfZhTx4RRv87D5fJgqrDHzR6ggICTnVSeGSJUtYtGgRK1asIDg4mMsuu4zc3NxS6xTvKthut5OTk8O+ffuK+s8fM2YMAwYMOKeulstyLt2l63L6FyurW+XLL7+8VNfR/fv3L+ru4dlnn6VOnToexe92u1mxYkWpPvrvv/9++vTpw5w5c0hJSeGyyy4rmlfy7+fJ3+Jc1Z6slpsJyybCkdJnH55oER7MvHGXMKxLM95cvIMR7//GoeOl/1ELUVsV7x65pMzMTOrXr09wcDBbtmzh119/9Xi7zZs3L+qqeMyYMecVW3ndJXfv3p0vv/wSoGh+Sb169Srq0njbtm3s3buXiy66qMxulcvqOrpbt25F8Q8ePNjjmPv168ebb75Z9H3t2rWA8bds2rQpYNTjV7fak/Sd+bDkX/Dbu+e9iSB/Oy/fEM/EG+NZn5rJVa8v4yep7hE+onjXyo888shp8wYMGIDT6aRjx4784x//oHv37tUa2xtvvMG0adPo2LEjH3/8Ma+//jpgDMg+ceJEunbtysGDB8usjhk7diwul4u4uDhuuukmpk+fTkBAALNmzSI2NpaEhAS2bNnCyJEjWb9+fdHN3RdeeIEnnniiQjGvXr2ajh070qFDB9555x0AHn30UR5//HF69ux51pZRVaF2da08Zwxs/gb+shkC61Qojh2Hsxg3Yw1bD2Vx96XRPHRlW4L9pTZMVB3pWvncZWdnExQUhFKKmTNn8tlnnzFv3jyrw6py0rVyoa53Q/4JWPtphTd14QVhzB3XkxHdjB46r3jtJxZs/LMSghRCVJbk5OSiJpdvv/02r732mtUh1Xi160wfYEpfyM2AcavAVjll2qqUdJ6Ys4Gth7K4ov0FPHVNDM0bBFfKtoUoJGf6wlNypl9ct3vh6A7Y9WOlbTIpqgHfTriExwe24+cdR7li4k+8tmArJ/Mq/8668G017SRM1DwV/TdS+5J+hyEQ2hB++U+lbtZht3Fv79Ys+mtv+sc04j8/7uCyV5cwa9VeXG75H1VUXGBgIEePHpXEL8qltebo0aMEBgae9zZqX/UOwM+vw8In4a4foVmXygmshN/3HuP5bzfx+94M2jeuwxOD2tPzwogq2ZfwDQUFBaSmppbZ/l2IQoGBgTRr1gyHw3HadE+rdzxK+kqpAcDrgB14X2td6skRpdSNwNOABv7QWo8wp48CCts9Pa+1/vBM+6qUpJ+XBf+OgahLYfiMim3rDLTWfLvuIC/+dwv7M3Lo2+4C7u/bhoTm0vmUEKJ6VVrSV0rZgW3AlUAqsAq4WWu9qdgybYDPgcu11seUUhdorQ8rpRoAq4FEjMIgGeiitT5W3v4qJekDLP4n/PQSjP0VLqjam2O5BS6m/ZzC24t3kJXnpHOLeozqEcWA2EYE+NmrdN9CCAGVeyO3K7BDa71La50PzASuLbHM3cBbhclca33YnN4fWKi1TjfnLQQGeHoQFdJtDDhCYPm/q3xXgQ47913Wml8ev5ynrunA0ZP5PDBzLT3+9SMv/neLDM8ohKgxPEn6TYF9xb6nmtOKawu0VUr9rJT61awO8nRdlFL3KKVWK6VWp6VV0hOwwQ0g8XZYPxvSd1fONs8iLNDB7T2jWfzXy/jwjq50blmf95bupNcri7l92kp+2HxIbvoKISzlySOmZfVsVDJz+QFtgMuAZsAypVSsh+uitX4PeA+M6h0PYvLMxeNg5Xvw8yS45vVK2+zZ2GyK3m0j6d02kgMZOcxcuZfPVu3jzg9X07ReEDclNWdQx8a0jizd97cQQlQlT870U4Hmxb43Aw6Uscw8rXWB1no3sBWjEPBk3apTpwl0ug3WfFJtZ/slNakXxF/6XcQvj13O27d0pkWDYCYu3Ebf136i/7+XMmnRNnYfOWlJbEII3+PJjVw/jBu5fYH9GDdyR2itNxZbZgDGzd1RSqkIYA2QwKmbt53NRX/HuJGbXt7+Ku1GbqHjB+GNBIi5Dq57p/K2WwEHM3P434Y/+e+GP1mVko7WEN+8HkMSmnBF+4bytK8Q4pxVdpPNq4BJGE02p2qtX1BKPYsxUsvXyujc+jWMm7Qu4AWt9Uxz3TuA/zM39YLWetqZ9lXpSR9gwRPwy5swdkWVt+Q5V39m5vL1H/uZu+YAmw4eB6BleDCXtomge6twEls2oFHd838QQwjhGyo16VenKkn6J4/C6/HQqneVttuvqF1pJ1i2/QjLtqexYudRTuYb3a42qx9EYsv6JEY1IDGqPm0vCMNm83wQCSFE7edp0veNvoJDwqHnA7D4edi5GFr3sTqiMrWKDKVVZCijekRR4HKz+eBxVqUcI3lPOj/vPMrctcbtkDqBfnRuWZ+kqAZ0aVmf+Gb1CPKX5wGEEGfnG2f6AAW58HZ3sPnBfT+DX8DZ16lBtNbsS89hVUo6q/ccY3VKOtsPnwDAz6aIbVr3tKuBiFDvOj4hRMVI9U5Zti+EGTdA36fg0r9UzT6qUUZ2Psl7jhUVAn+kZpLvdAMQHRFCl5b1iwqC1pEh5zSuqBDCu0jSL8/MW2DnjzBuJdRrfvblvUie08WG/ZmsTjlWVC10LLsAgLBAPzo0rkNMk7rENKlDTNM6XBgZKoO/C1FLSNIvT8ZeeLMrtLkCbvqk6vZTA2it2XXkJKtT0lmXmsnGA8fZ8udxcguMqwGHXdGiQTDREaG0igzhwgtCaXNBKK0iQqkb7DjL1oUQNYncyC1PvRbQ62H48TnYvshI/rWUUorWkaG0jgzlpiRjmtPlZveRk2YBkEXKkZPsPnKSpdvTiqqGAOoFO2jZIJiW4SFEhZvvEcZ7eIi/VBUJ4aV870wfwJkHb18MaLjvF3AEVe3+vIDLrdmbns32Q1nsOZpNytGTRe8HMnIo3mVQaIAfLcODiQoPoWV4sPkKISo8hAvCAqQ5qRAWkDP9M/ELgEGvwcdD4Mfnof8LVkdkObtNER0RQnRESKl5+U43qceySxUGmw4e5/uNf+IsViIEOmy0bGAUBs3qB9O0fhBN6wVx4QWhtGgQjL+f3EMQwkq+mfTBaKvf5XZY8Ra0GwQte1gdUY3l72creoagJKfLzYGMXPaknyTlaDZ7jhjvu4+cZNn2I+QUuE5bvl6wg4ZhgUWFQfH3ZvWCiAiVKwUhqpJvVu8UyjsBk3uAUjDmZwiQXi8rk9aaY9kFpB7LZvuhE+zPyOFwVi5/ZuaxPyOH/ceyOZ57+uDy/n42mtQNJCI0gPBQfxqEBBAR6k+DEH8iwwJoWCeQhmGBXFAngECHPJAmRCGp3vFEQKjRCdu0q2DhP+Dqqh9wxZcopWgQYiTsjs3KHkIyK7fALAByTns/eiKf3UdOkrznGOkn8ylrGIKwAD/qhTioF+RPvWAH9YL9qRfkoH6wg7rB/tQPdpw2vV6wP3WDHNjlSkL4MN9O+mBU61w8Dla8CRddBW2utDoinxIW6KBdIwftGtUpdxmXW5ORnU/aiTwOH8/j0PFcDmflkZaVR2ZOARnZ+WTkFJB6LIeM7HwycwrKLCQK1Qn0o37IqYKgXrCD+sH+1AlymNMc1A069QoLdFAnyI8gh11aLQmv59vVO4UKcmHK5XDikNFFQ1ij6t2/qFRutyYr18kxszA4lp1PZrbxnpF9qpA4ll1AZuHnk/lk5Tk50/8OdpuiTqAfYYEOwgL9qGO+FxYKYYEOc74fgQ47AX42AvzMd/N7oMNGoMNOsL8fwf7GNClIRGWQ6p1z4QiEYdPg3d4w5164dQ7YpJWJt7LZFHWDHef8gJnLrcnKLSAju4DMnFOv47kFZOU6ycot4HiO+W5+35uezfEcc36e8+w7KRmrgiCHnSCzEDj18iOo2Pcghx8hAXZjmlloBJgFSJDDXlTI+NkVDrvNLGDsBPrZCXDYpHARRSTpF4q8CAa+BN9MgOWvQa9HrI5IVDO7TZnVPf7ntb7LrTmR5+REnpO8Ahd5Tje55nue001egYtcc1pOvovsfBfZ+U7z3UWO+TmnwPh+5ERe0eccc9mKDLFcVBA4bDjshS+Fn814t9uMz3abwq/Ydz+bwm5X2JTCroxC1a6M+TabwqbArlSp6Q67DYdN4fAztuHvZ+yz8HPhfgtj8Sv6bO7LZm6r8LMy4vCzFcZm7MfPduoYpGA7O0n6xXUeCSnLYPE/oWlije2CWdRMdpsqug9QFbTW5DndnMxzklPgIrfAKEByzc95ThdOt6bA5SavwE2u89QyRYWOuWyBy02BW1PgdON0uylwadxa43RpXG5NrtOFy33qe4HbjdZGweZyG8sa75z67Na49Kn5Ba7qrzouLAz87TbsdqOgUMoomGzmu1IKm8347mdTBPjZ8fczrob8i1fJmd8LCxabUhSWKQrjc2ERo8ztGvOM/yhO36/N3EZRDMXjMgvPyLAAru7YpEr/RpL0i1PKGED9zw3w5Z1w7zKo29TqqIQAjEQRaFbleAOtNU6z4Mh3GQWN02UUSsbr9M9Ol5t8cxmXLl2IuNzG/RqnW+Nyu833wn2Y23C7i/bndGk0RsGktcbtNgqowu8us5AzrsRc5DvdZOU6OerMN76bhWdhoVb0EKI2xoEtvB9qfC6cpU991hTt3+XhJVpC83qS9Kudfwjc9DG8dxl8MRpu/w7s0vmYEOdKKWVW30AQ3lFQVSWtT10ZubVROBS/YtJaV0v1lNytLEtEGxj8BqSuhEVPWx2NEKIWUOa9CeNGu72oFVdYoFElWPgcSVWTpF+e2Osh6W6j/f7mb62ORgghKoUk/TPp/wI06QRf3QMH/7A6GiGEqDBJ+mfiFwDDP4Og+vDpTZCZanVEQghRIZL0z6ZOY7jlc8g/CTNuhNzjVkckhBDnTZK+JxrGwI0fwZGt8PlIcBVYHZEQQpwXSfqeat0Hrp4EuxbDtw9yxk5ahBCihpJ2+uei823GwOpLX4bAetDveZDHvoUQXkSS/rnq83+Qm2E05fQPhT6PWx2REEJ4TJL+uVIKBrwE+dnw04vGE7w9J1gdlRBCeESS/vmw2YwndguyjRG3/IMh6S6roxJCiLOSpH++bHYY+h4U5MD8v4IjBBJutjoqIYQ4I2m9UxF2BwybDtG9Yd5Y2DTP6oiEEOKMJOlXlCMQbv4MmnWF2XfCtgVWRySEEOWSpF8Z/EOMp3YbdoDPb4NdS6yOSAghyiRJv7IE1jXG1m3QCj4dLolfCFEjSdKvTCHhMOobaBAtiV8IUSNJ0q9sIRHFEv9NsHOx1REJIUQRSfpVoSjxt4bPhsOORVZHJIQQgCT9qlOY+MPbGGf8v39kdURCCCFJv0qFhBsDq0f3gq/vh0XPgNttdVRCCB/mUdJXSg1QSm1VSu1QSj1WxvzRSqk0pdRa83VXsXmuYtO/rszgvUJgHRjxOXQZDcsnwpd3QkGu1VEJIXzUWbthUErZgbeAK4FUYJVS6mut9aYSi87SWo8vYxM5WuuEiofqxewOoy/+Bq1g4ZNwfD8M/9SoAhJCiGrkyZl+V2CH1nqX1jofmAlcW7Vh1UJKQc8HYNiHxiDr7/eFI9utjkoI4WM8SfpNgX3Fvqea00q6Xim1Tik1WynVvNj0QKXUaqXUr0qpIWXtQCl1j7nM6rS0NM+j90YxQ2DUt5B3At6/AlKWWx2REMKHeJL0yxoaquRYgd8AUVrrjsAi4MNi81porROBEcAkpVTrUhvT+j2tdaLWOjEyMtLD0L1Y8yS4+wcIvQA+GgJ/zLQ6IiGEj/Ak6acCxc/cmwEHii+gtT6qtc4zv04BuhSbd8B83wUsATpVIN7ao34U3LkAWnSHOffC4n/JuLtCiCrnSdJfBbRRSkUrpfyB4cBprXCUUo2LfR0MbDan11dKBZifI4CeQMkbwL4rqD7c+hUk3GKMwjXnXnDmnX09IYQ4T2dtvaO1diqlxgPfA3ZgqtZ6o1LqWWC11vprYIJSajDgBNKB0ebq7YF3lVJujALmxTJa/fg2P3+49i2oHw2Ln4fMVLjpEwhuYHVkQohaSOkaVqWQmJioV69ebXUY1lj3hTEYS2gjuPFDaNrZ6oiEEF5CKZVs3j89I3kitybpOAzu+B+gYWp/+PUdeYJXCFGpJOnXNE27wD0/Qas+8L+/wfRBcHSn1VEJIWoJSfo1UUg4jJgFQybD4Y0wuSf89q607hFCVJgk/ZpKKUgYAWN/Mzps+++jMPMWyE63OjIhhBeTpF/T1WlsnPX3/xdsXwDv9oJ9K62OSgjhpSTpewOl4OKxcOf3oGzGTd4fngNnvtWRCSG8jCR9b9K0C4xZBvEjYNmrMKUP/Lne6qiEEF5Ekr63CawLQ96C4Z/BicPwXh9Y+iq4nFZHJoTwApL0vVW7q2Dsr9BuEPz4HEztB2nbrI5KCFHDSdL3ZiHhxpO7N0yF9F3w7qWwfJKc9QshyiVJvzaIvd4462/dFxY9Be9fbgzUIoQQJUjSry3CGsHwGTBsOhw/YNT1L3wKCnKsjkwIUYNI0q9NlIKY62DcSki4GX6eBJN7wO5lVkcmhKghJOnXRsENjO6aR84D7YYPr4avJ0BOhtWRCSEsJkm/Nmt1Gdy3AnrcD2s+hre6weZvrI5KCGEhSfq1nX8w9Hse7voBQiJh1q3CeV9IAAAS2klEQVTGK+tPqyMTQlhAkr6vaNoZ7lkMfZ+EbQvgra6Q/KH01y+Ej5Gk70vsDrj0r3DfL9AwFr6ZAO/3hVQfHalMCB8kSd8XRVwIo76F696F4/uNxD93LJw8anVkQogqJknfV9lsED8c7k+Gng/CullGlc/KKdJ7pxC1mCR9XxcQBlc+A/cuhYi28N3D8GYibPhSRuoSohaSpC8MDWPg9u/gli8hoA7MvsPotz812erIhBCVSJK+OEUpaHMF3PsTDP4PpO82+vH56h7I3G91dEKISiBJX5Rms0PnkTDhd7jkL7BxLvynCyx6GrIOWR2dEKICJOmL8gWEwRVPwfhVRv/9yyfBpFiY/1c4kWZ1dEKI8yBJX5xd/ZZGn/33J0PCCEieDm90guX/hoJcq6MTQpwDSfrCc+Gt4ZrXjb77oy4xqnveSoINX0lLHyG8hCR9ce4i2sCImXDbXLOlz+1GS589v1gdmRDiLCTpi/PXuo/Rvn/wf+DYHpg2ED65Hg6ssToyIUQ5JOmLiilq6bMGrnwW9ifDe5fB5yMhbavV0QkhSpCkLyqHfzD0fAAe+AN6PwY7foC3u8Oc++BYitXRCSFMkvRF5QqsC30ehwfWQfexsPEr+E8izH9Y+vAXogaQpC+qRkg49H/BqPbpfBskT4PXE2Dhk5CdbnV0QvgsSfqiatVpAlf/23jAq8Ng+PkNeD0elrwEeVlWRyeEz5GkL6pHg1Yw9D1jAJfoXrDkn8aZ/6/vgDPP6uiE8BmS9EX1atgBhs8wxuy9oD38729Gnf/az8Dtsjo6IWo9SfrCGs0SYdQ3cNscCG4Ac8fA5J6w5Tt5uleIKiRJX1hHKWh9OdyzBIZ9CO4CmHkzfNAPtv5XBm0Xogp4lPSVUgOUUluVUjuUUo+VMX+0UipNKbXWfN1VbN4opdR28zWqMoMXtYRSEDMExv4G17wBxw/AZ8NhSh/YtcTq6ISoVZQ+y6W0UsoObAOuBFKBVcDNWutNxZYZDSRqrceXWLcBsBpIBDSQDHTRWh8rb3+JiYl69erV53UwopZwFcD6L2DxPyFzHzRNhIvHQYchxti+QohSlFLJWuvEsy3nyf9BXYEdWutdWut8YCZwrYdx9AcWaq3TzUS/EBjg4brCV9kdRhfO41fDwJchN8Po1G1yD9g4R6p9hKgAT5J+U2Bfse+p5rSSrldKrVNKzVZKNT/HdYUozREI3e6FcSvh+g9Au+GL0ZL8hagAT5K+KmNayTqhb4AorXVHYBHw4Tmsi1LqHqXUaqXU6rQ0GZFJlGCzQ9wNMHZF6eS/4Stp6inEOfAk6acCzYt9bwYcKL6A1vqo1rrwCZspQBdP1zXXf09rnai1ToyMjPQ0duFrykr+s2+Ht7rB2k+NewFCiDPyJOmvAtoopaKVUv7AcODr4gsopRoX+zoY2Gx+/h7op5Sqr5SqD/Qzpwlx/oon/2HTwS8Q5t4Hb3Q2unnIKbedgBA+76xJX2vtBMZjJOvNwOda641KqWeVUoPNxSYopTYqpf4AJgCjzXXTgecwCo5VwLPmNCEqzmaHmOtgzDIY8TnUaw4L/wETO8C3D8HhLVZHKESNc9Ymm9VNmmyKCjm4Dla+C+u+AFcetO4LPe6HVpcZzwMIUUtVZpNNIbxH445w7Vvwl81w+RPw53r4eAi8cyn8MUvq/YXPk6QvaqeQcOj1CDy4Hga/Ca58mHOP0a3zL/+B3ONWRyiEJSTpi9rNEWgM4jL2VxjxhdHF84In4N+xsOhpGc1L+BxJ+sI32GzQth+M/hbu/hFa94GfX4dJcTBvvNz0FT7Dz+oAhKh2TbvAjR/C0Z3w69uwZgas+Rha9YFuY6BNP+njR9Ra8i9b+K7w1jDoNXhoI1z+D0jbCp/dZDzp+8dMuekraiVJ+kKEhEOvh+HBdTB0itG0c869RtXPstdkIHdRq0jSF6KQ3QEdb4QxPxsPe0W2gx+ehYnt4ZsHpN5f1ApSpy9ESTYbtO1vvA5tgt/eMap7kqcbI311H2s89CX1/sILyRO5Qnji5FFIngYrp8CJPyG8DXQfA/E3g3+I1dEJIU/kClGpiur918PQ9yEgFOb/1aj6WfgkZKZaHaEQHpEzfSHOh9awb6XR5HPz14CCDoONqp/mXa2OTvggT8/0pU5fiPOhFLToZrwy9hrVPskfGiN6NU2E7vdBh2uNm8NC1CBSvSNERdVrAf2eg79sgqteNcb0/fJOmNQRlk2UJp+iRpHqHSEqm9sNOxYZVT+7FoNfEMQOhc6jjKof6eJZVAGp3hHCKoX9/LTtZzT5XPkurJ8Na2dAZHvoPBLih0NwA6sjFT5IzvSFqA55J2DDl/D7h7A/Gez+0G4QxI8w2v7b5fxLVIyc6QtRkwSEQpdRxuvPDfD7R7D+c+PGb2hDiBsGCSOgYYzVkYpaTs70hbCKMx+2fw9rPzPe3U5oFGec/ccNg9BIqyMUXsTTM31J+kLUBCePGNU/az+Fg2vB5gdtBxj1/637SvWPOCtJ+kJ4q8ObYc0nRn8/2UcgrDEk3AKdboUG0VZHJ2ooSfpCeDtnPmz7n1H/v/MH0G6I7gWdboP214AjyOoIRQ0iN3KF8HZ+/kbXDh0GQ+Z+o+pnzUfw1d0QUMdo+59wKzRLlLb/wmNypi+EN3G7Yc9yY4jHTfPAmQMRF0GnW6DjcAhraHWEwiJSvSNEbZd73GjyueYTSF0Jyg5trjTq/9sOMK4UhM+Q6h0harvAOqfa/qdtM574/WOmcR8gOBzibjSuABrFWR2pqEHkTF+I2sTlNG76rvkEtv4X3AXQpJPR9DP2BqOgELWSVO8I4etOHjWe+v39Izi8CRzBEDPUKACk47daR5K+EMKgNez/3ej3Z8OXkH/CGPQ9/maIGQL1o6yOUFQCSfpCiNLyThg3f3//yLj5C9Asyej3J2YoBNWzNj5x3iTpCyHO7Nge2PgV/DEL0jaDPQDaX20UAK36gM1udYTiHEjSF0J4Rmujv5+1n8L6LyDnmNH1Q/xwo/O3yLZWRyg8IElfCHHunHlGk8+1n8L2haBdUv3jJSTpCyEqJuuQ0fpnzYzTq3863QbRvY0RwkSNIUlfCFE5yqr+qR9lNP1MuAXCGlkdoUCSvhCiKhTkwpZvIXk6pCwzun648ApIuNno+kF6/rSMdMMghKh8jkCIu8F4HdlxquuHL74H/zCjR9D4m6FlT6n+qaHkTF8IUTFuF6QsN8f8nQf5WVCvhdHyJ364DPxSTaR6RwhR/fKzYct84wpg1xJAQ8tLjNY/Ha41BogXVcLTpO/R9ZdSaoBSaqtSaodS6rEzLHeDUkorpRLN71FKqRyl1Frz9Y7nhyCE8Dr+wdBxGIycCw9tgMv/AVkHYd5YeLUtzLnPKAzcLqsj9VlnrdNXStmBt4ArgVRglVLqa631phLLhQETgN9KbGKn1jqhkuIVQniLus2g18Nw6V9h30rj7H/DV/DHp8bDX3E3GN0/N4qTzt+qkSdn+l2BHVrrXVrrfGAmcG0Zyz0HvAzkVmJ8QghvpxS06AaD34BHtsMN06BxAvw6Gd69FN6+GJZNhIx9VkfqEzxJ+k2B4r9GqjmtiFKqE9Bca/1tGetHK6XWKKV+Ukpdev6hCiG8niPIGNt3xEz46zYY9JrRx/8Pz8CkWJh2ldEcNOeY1ZHWWp402Szruqvo7q9Sygb8GxhdxnIHgRZa66NKqS7AXKVUjNb6+Gk7UOoe4B6AFi1aeBi6EMKrhYRD0l3GK303rJ8N62bBNw/Ad49Am34QN8xs/x9odbS1xllb7yilLgae1lr3N78/DqC1/pf5vS6wEzhhrtIISAcGa61Xl9jWEuDhktOLk9Y7Qviwwqd/130BG2bDiUMQUBc6XGPU/0ddIr1/lqPSmmwqpfyAbUBfYD+wChihtd5YzvJLMBO7UioSSNdau5RSrYBlQJzWOr28/UnSF0IARguf3T8ZBcDmr43BX8KaQNz1xhVAo45yA7iYSnsiV2vtVEqNB74H7MBUrfVGpdSzwGqt9ddnWL0X8KxSygm4gDFnSvhCCFHEZofWlxuvQa/Btv8aBcCvk+GX/0B4G6MFUOwNEHGh1dF6DXk4SwjhXbLTYdM8Y+jHlOWAhsbxRvKPHWo0FfVB8kSuEKL2O37AGP5xw5ewPxlQEH2p0f9P+8E+9QSwJH0hhG9J32VU//zxGRzbDY5gaH+N0f9PVC+w1+7+JSXpCyF8k9bGE8B/fGaMAZybCcERRg+gsddDix61sgdQSfpCCFGQC9u/N6qAtv4PnDlGFxAx1xkFQNMutaYFkCR9IYQoLu+EMf7vhq9gx0Jw5UO9lsbN39jroWGsVxcAkvSFEKI8ORlGF9AbvjR6/dQuiGh7qgVQRBurIzxnkvSFEMITJ4+YTUC/gj0/A9p48Cv2eqMAqOcdXcNI0hdCiHN1/ABsnGs2ATXzULOuRgEQM6RGDwIvSV8IISoifbf5DMBXcGg9oIy+f+JuMJ4BCG5gdYSnkaQvhBCVJW2rcfa/4Us4ugNsftC6r3EFcNFAo3toi0nSF0KIyqY1/LnO6AZ6w1dwPBXsAXDhFcYYwBcNgMC6loRWaR2uCSGEMCll9PPTOB6ueAZSVxr3ADbNg63zwe5vXAHEDIF2gyAgzOqIS5EzfSGEqCi327jxW1gAHE8Fv0Cj6if2BuNKoIoHgpHqHSGEsILbDamrYP0XRjcQ2UfBP9QYCazDtdDmSvAPqfTdStIXQgiruQqMgWA2fW08DJZ9BPyC4MK+0GEItO1faTeBJekLIURN4nLC3l+MAmDzN3DiT+MeQKs+5k3ggRVqBipJXwghaiq327gJvOnrU/cAbH5GV9DDpp/XJqX1jhBC1FQ2G7Tobrz6vwAHfjeSv63qU7IkfSGEsJJSRhfPTbtUy+5q30gCQgghyiVJXwghfIgkfSGE8CGS9IUQwodI0hdCCB8iSV8IIXyIJH0hhPAhkvSFEMKH1LhuGJRSacAeq+MoIQI4YnUQlUyOyXvUxuOSY6p8LbXWkWdbqMYl/ZpIKbXakz4tvIkck/eojcclx2Qdqd4RQggfIklfCCF8iCR9z7xndQBVQI7Je9TG45JjsojU6QshhA+RM30hhPAhkvRLUEqlKKXWK6XWKqVWm9MaKKUWKqW2m+/1rY7zbJRSU5VSh5VSG4pNK/M4lOENpdQOpdQ6pVRn6yIvXznH9LRSar/5e61VSl1VbN7j5jFtVUr1tybqM1NKNVdKLVZKbVZKbVRKPWBO99rf6gzH5LW/lVIqUCm1Uin1h3lMz5jTo5VSv5m/0yyllL85PcD8vsOcH2Vl/KfRWsur2AtIASJKTHsZeMz8/BjwktVxenAcvYDOwIazHQdwFfBfQAHdgd+sjv8cjulp4OEylu0A/AEEANHATsBu9TGUEWdjoLP5OQzYZsbutb/VGY7Ja38r8+8dan52AL+Zf//PgeHm9HeA+8zPY4F3zM/DgVlWH0PhS870PXMt8KH5+UNgiIWxeERrvRRILzG5vOO4FvhIG34F6imlGldPpJ4r55jKcy0wU2udp7XeDewAulZZcOdJa31Qa/27+TkL2Aw0xYt/qzMcU3lq/G9l/r1PmF8d5ksDlwOzzeklf6fC32820Fcppaop3DOSpF+aBhYopZKVUveY0xpqrQ+C8Q8auMCy6CqmvONoCuwrtlwqZ/6ftKYZb1Z1TC1W9eZ1x2RWAXTCOIusFb9ViWMCL/6tlFJ2pdRa4DCwEOOKJENr7TQXKR530TGZ8zOB8OqNuGyS9EvrqbXuDAwEximlelkdUDUo6wzEW5p1TQZaAwnAQeA1c7pXHZNSKhT4EnhQa338TIuWMa1GHlcZx+TVv5XW2qW1TgCaYVyJtC9rMfO9xh6TJP0StNYHzPfDwByMH/dQ4SW0+X7YuggrpLzjSAWaF1uuGXCgmmM7L1rrQ+b/jG5gCqeqBbzmmJRSDozkOENr/ZU52at/q7KOqTb8VgBa6wxgCUadfj2llJ85q3jcRcdkzq+L51WTVUqSfjFKqRClVFjhZ6AfsAH4GhhlLjYKmGdNhBVW3nF8DYw0W4Z0BzILqxZquhL12ddh/F5gHNNwsxVFNNAGWFnd8Z2NWc/7AbBZaz2x2Cyv/a3KOyZv/q2UUpFKqXrm5yDgCox7FYuBG8zFSv5Ohb/fDcCP2ryrazmr7yTXpBfQCqMVwR/ARuDv5vRw4Adgu/newOpYPTiWzzAuoQswzjruLO84MC5F38Koo1wPJFod/zkc08dmzOsw/kdrXGz5v5vHtBUYaHX85RzTJRiX/euAtebrKm/+rc5wTF77WwEdgTVm7BuAJ83prTAKqB3AF0CAOT3Q/L7DnN/K6mMofMkTuUII4UOkekcIIXyIJH0hhPAhkvSFEMKHSNIXQggfIklfCCF8iCR9IYTwIZL0hRDCh0jSF0IIH/L/UHlEm/GNOm4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bc85a306a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,\n",
    "        max_depth=6,\n",
    "        num_class=3,\n",
    "        min_child_weight=1,\n",
    "        gamma=0.2,nthread=4,\n",
    "        subsample=0.8,reg_alpha=0.01,reg_lambda=2,\n",
    "        colsample_bytree=0.8,\n",
    "        objective= 'multi:softprob', \n",
    "        seed=20)\n",
    "modelfit(xgb5, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 确定了最优 n_estimators 后 再对 subsample 和 colsample_bytree 进行调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.6, gamma=0.2, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=6, min_child_weight=1, missing=None,\n",
       "       n_estimators=317, n_jobs=1, nthread=4, objective='multi:softprob',\n",
       "       random_state=0, reg_alpha=0.01, reg_lambda=2, scale_pos_weight=1,\n",
       "       seed=27, silent=True, subsample=0.9),\n",
       "       fit_params=None, iid=False, n_jobs=4,\n",
       "       param_grid={'subsample': [0.6, 0.7, 0.8, 0.9], 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test6 = {\n",
    "    'subsample':[i/10.0 for i in range(6,10)],\n",
    "    'colsample_bytree':[i/10.0 for i in range(6,10)]\n",
    "}\n",
    "gsearch6= GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=317, max_depth=6,\n",
    "                                        min_child_weight=1, gamma=0.2, subsample=0.9, colsample_bytree=0.6,reg_alpha=0.01,reg_lambda=2,\n",
    "                                        objective= 'multi:softprob', nthread=4, scale_pos_weight=1,seed=27), \n",
    "                       param_grid = param_test6, scoring='neg_log_loss',n_jobs=4,iid=False, cv=5)\n",
    "gsearch6.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: -0.58653, std: 0.00407, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58459, std: 0.00403, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58386, std: 0.00445, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58324, std: 0.00409, params: {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       "  mean: -0.58637, std: 0.00399, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58476, std: 0.00382, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58484, std: 0.00412, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58386, std: 0.00376, params: {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "  mean: -0.58739, std: 0.00403, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58580, std: 0.00443, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58456, std: 0.00468, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58426, std: 0.00452, params: {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "  mean: -0.58674, std: 0.00398, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58636, std: 0.00426, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58568, std: 0.00434, params: {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       "  mean: -0.58434, std: 0.00428, params: {'colsample_bytree': 0.9, 'subsample': 0.9}],\n",
       " {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       " -0.58323663779650681)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch6.grid_scores_, gsearch6.best_params_, gsearch6.best_score_  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在测试集上进行训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb7 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=317, \n",
    "        max_depth=6,\n",
    "        min_child_weight=1,\n",
    "        gamma=0.2,\n",
    "        reg_alpha=0.01, \n",
    "        reg_lambda=2,\n",
    "        subsample=0.6,\n",
    "        colsample_bytree=0.9,\n",
    "        nthread=4,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.83\n"
     ]
    }
   ],
   "source": [
    "xgb7.fit(X_train, y_train)\n",
    "y_train_pre = xgb7.predict(X_train)\n",
    "print (\"Accuracy : %.2f\" % metrics.accuracy_score(y_train, y_train_pre))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 预测输出测试集的标签,保存 csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = xgb7.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds=np.where(preds==0, 'low',np.where(preds==1, 'medium', 'high'))\n",
    "out_df = pd.DataFrame(preds)\n",
    "\n",
    "out_df.columns = [\"interest_level\"]  \n",
    "out_df.to_csv(\"submission.csv\", index = False, header = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74629</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74630</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74631</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74632</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74633</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74634</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74635</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74636</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74637</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74638</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74639</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74640</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74641</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74642</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74643</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74644</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74645</th>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74646</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74647</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74648</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74649</th>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74650</th>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74651</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74652</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74653</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74654</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74655</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74656</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74657</th>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74658</th>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74659 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      interest_level\n",
       "0               high\n",
       "1                low\n",
       "2               high\n",
       "3               high\n",
       "4               high\n",
       "5               high\n",
       "6               high\n",
       "7             medium\n",
       "8               high\n",
       "9               high\n",
       "10              high\n",
       "11              high\n",
       "12              high\n",
       "13              high\n",
       "14              high\n",
       "15              high\n",
       "16              high\n",
       "17              high\n",
       "18              high\n",
       "19            medium\n",
       "20              high\n",
       "21              high\n",
       "22              high\n",
       "23            medium\n",
       "24              high\n",
       "25            medium\n",
       "26              high\n",
       "27            medium\n",
       "28              high\n",
       "29              high\n",
       "...              ...\n",
       "74629         medium\n",
       "74630           high\n",
       "74631           high\n",
       "74632           high\n",
       "74633           high\n",
       "74634           high\n",
       "74635           high\n",
       "74636           high\n",
       "74637           high\n",
       "74638           high\n",
       "74639           high\n",
       "74640           high\n",
       "74641           high\n",
       "74642           high\n",
       "74643           high\n",
       "74644           high\n",
       "74645            low\n",
       "74646           high\n",
       "74647           high\n",
       "74648         medium\n",
       "74649         medium\n",
       "74650            low\n",
       "74651           high\n",
       "74652           high\n",
       "74653           high\n",
       "74654           high\n",
       "74655           high\n",
       "74656           high\n",
       "74657            low\n",
       "74658           high\n",
       "\n",
       "[74659 rows x 1 columns]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
